using System;
using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;
using System.Xml;
using System.Xml.XPath;
using System.Net;
using CKS.EBE.Logging;
using CookComputing.XmlRpc;
using Microsoft.SharePoint;
using HtmlAgilityPack;
using System.Threading;
using System.Diagnostics;
using CKS.EBE.API;

namespace CKS.EBE.Linkbacks
{
	public class LinkbackPinger
	{
		private readonly SPWeb _CurrentWeb;
		private readonly Logger _Logger;

		public LinkbackPinger(SPWeb web)
		{
			_CurrentWeb = web;
			var blogSettings = new BlogSettings(_CurrentWeb);
			var level = (LoggingLevel) blogSettings.LoggingLevel;
			_Logger = new Logger(level);
			_Logger.Verbose("LinkbackPinger Constructor", "Creating LinkbackPinger");
		}

		#region Trackback/Pingback constants
		private const int MaxExcerptLengthSubmittedWithLinkback = 64;

		// When there are a lot of items in a list, SPList.Items.Add can get very slow. Using SPWeb.ProcessBatchData() instead.
		// In addition, in order to use internal field names we use this method (display names can vary with language).
		private const string AddLinkbackHistoryCamlBatchFormat =
			 @"<?xml version=""1.0"" encoding=""UTF-8""?>
            <ows:Batch>
            <Method ID=""B2"">
            <SetList>{0}</SetList>
            <SetVar Name=""ID"">New</SetVar>
            <SetVar Name=""Cmd"">Save</SetVar>
            <SetVar Name=""urn:schemas-microsoft-com:office:office#Title"">{1}</SetVar>
            <SetVar Name=""urn:schemas-microsoft-com:office:office#PostTitle"">{2}</SetVar>
            <SetVar Name=""urn:schemas-microsoft-com:office:office#LinkbackType"">{3}</SetVar>
            <SetVar Name=""urn:schemas-microsoft-com:office:office#SentToUrl"">{4}</SetVar>
            <SetVar Name=""urn:schemas-microsoft-com:office:office#PingUrl"">{5}</SetVar>
            <SetVar Name=""urn:schemas-microsoft-com:office:office#SendResult"">{6}</SetVar>
            <SetVar Name=""urn:schemas-microsoft-com:office:office#SendErrorMessage"">{7}</SetVar>
            </Method>
            </Batch>"; // {0} = Linkbacks Sent list GUID, {1} = caption for list item, 
		// {2} local post lookup value (ID number), {3} Trackback or Pingback,
		// {4} = The permalink of the foreign post (if available), {5} = The ping URL of the foreign post, {6} = The result type (success, fail, etc), 
		// {7} = The error message in sending the linkback (if any)

		private const string SearchLinkbackHistoryListCamlQueryFormat =
			 @"<Where>
                <And>
                    <Eq>
                        <FieldRef Name=""PostTitle"" LookupId=""True"" />
                        <Value Type=""Lookup"">{0}</Value>
                    </Eq>
                    <Eq>
                        <FieldRef Name=""PingUrl"" />
                        <Value Type=""Text"">{1}</Value>
                    </Eq>
                </And>
            </Where>"; // {0} = ID of local (originating) post, {2} = Ping URL (XML escaped) of foreign post

		#endregion

		#region Trackback/Pingback methods

		public void SendLinkbackPings(SPListItem postListItem)
		{
			try
			{
				_Logger.Verbose("SendLinkbackPings", "Creating Blogsettings for web " + _CurrentWeb.Url);
				var bs = new BlogSettings(_CurrentWeb);

				if (bs.PingbacksEnabled == false)
				{
					_Logger.Verbose("SendLinkbackPings", "Pingbacks are disabled");
					return;
				}
				_Logger.Verbose("SendLinkbackPings", "for postID " + postListItem.ID);
				var linkbackThreadParameters = new LinkbackThreadParameters(postListItem.ID, _CurrentWeb.Url, bs);

				var thread = new Thread(SendLinkbackPingsInternal);
				thread.Start(linkbackThreadParameters);
			}
			catch (Exception ex)
			{
				Trace.Write(ex);
			}
		}

		private void SendLinkbackPingsInternal(object parameters)
		{
			var parameter = (LinkbackThreadParameters)parameters;
			using (var oSite = new SPSite(parameter.SiteUrl))
			using (SPWeb oWeb = oSite.OpenWeb())
			{
				try
				{
					string postsListName = Localization.GetListName(Localization.BlogList.Posts, oWeb.Language);
					SPList postsList = EBE.Helper.GetListByFolderName(oWeb, postsListName);
					SPListItem oItem = postsList.GetItemById(parameter.ItemId);
					_Logger.Verbose("SendLinkbackPingsInternal", "Got list and item");
					SendLinkbackPingsForItem(oItem, parameter);
				}
				catch (Exception ex)
				{
					_Logger.Error("SendLinkbackPingsInternal", ex.ToString());
				}
			}

			// write to log here, because after the method the new thread ends
			_Logger.WriteEntriesToLog("TrackbackLog.txt");
		}

		private void SendLinkbackPingsForItem(object postSPListItem, LinkbackThreadParameters parameter)
		{
			var postListItem = postSPListItem as SPListItem;

			// Ensure post is published:
			if (postListItem == null || postListItem.Level != SPFileLevel.Published)
			{
				_Logger.Verbose("SendLinkbackPingsForItem", "Post is null or not published");
				return;
			}

			// If approval is enabled and the post is not approved, do not send pings:
			if (postListItem.ModerationInformation != null && postListItem.ModerationInformation.Status != SPModerationStatusType.Approved)
			{
				_Logger.Verbose("SendLinkbackPingsForItem", "Post moderation status is not approved");
				return;
			}

			// only pingback, if the post is published
			var postPublishedDate = (DateTime)postListItem[SPBuiltInFieldId.PublishedDate];
			if (postPublishedDate > DateTime.Now)
			{
				_Logger.Verbose("SendLinkbackPingsForItem", string.Format("the posts publishing date ({0}) is in the future. Don't pingback, but remember the post id", postPublishedDate));

				// read ids of listitems which need to be pinged later
				string property = postListItem.Web.Properties["EBE_PingbackIDs"];
				var properties = string.IsNullOrEmpty(property) ? new List<string>() : new List<string>(property.Split(';'));
				if (!properties.Contains(postListItem.ID.ToString()))
				{
					// add the item id, so we can ping later
					property += string.Format(";{0}", postListItem.ID);
					postListItem.Web.Properties["EBE_PingbackIDs"] = property;
					_Logger.Verbose("SendLinkbackPingsForItem", "EBE_PingbackIDs are " + property);
					postListItem.Web.Properties.Update();
					postListItem.Web.Update();
				}

				return;
			}

			// Initialization:
			var encounteredPingUrls = new Dictionary<string, bool>();
			SPWeb currentWeb = postListItem.Web;
			SPList parentList = postListItem.ParentList;
			string postBodyHtml = Convert.ToString(postListItem[SPBuiltInFieldId.Body]);

			// Initialize the information about the local post to submit with the ping:
			var localPostInfoToSubmit = new LocalPostInfo { WebUrl = currentWeb.Url, ItemId = postListItem.ID, Title = postListItem.Title };
			localPostInfoToSubmit.Url = string.Format("{0}/{1}/Post.aspx?ID={2}", currentWeb.Url, parentList.RootFolder.Url, localPostInfoToSubmit.ItemId);
			//localPostInfoToSubmit.Url = EBE.Helper.GetFriendlyPostUrl(currentWeb.Url,postListItem);
			localPostInfoToSubmit.Excerpt = GetExcerpt(parentList.Fields[SPBuiltInFieldId.Body].GetFieldValueAsText(postBodyHtml));
			localPostInfoToSubmit.BlogName = currentWeb.Title;
			_Logger.Information("SendLinkbackPingsForItem", localPostInfoToSubmit.ToString());

			#region Direct Ping
			try
			{
				if (parentList.Fields.ContainsField("Trackback ping URLs"))
				{
					// Ping each of the direct ping URLs listed in the field
					string pingUrls = Convert.ToString(postListItem["Trackback ping URLs"]).Replace("\r\n", "\n").Trim();
					_Logger.Verbose("SendLinkbackPingsForItem", "PingUrls from 'Trackback ping URLs' field:" + pingUrls);
					if (string.IsNullOrEmpty(pingUrls) == false)
					{
						var publishedDate = (DateTime)postListItem[SPBuiltInFieldId.PublishedDate];
						if (publishedDate > DateTime.Now)
						{
							_Logger.Verbose("SendLinkbackPingsForItem", string.Format("the posts publishing date ({0}) is in the future. Don't pingback", publishedDate));
							CreateTimerJob(postListItem);
							// don't process further trackbacks now
							return;
						}

						foreach (string directPingUrl in pingUrls.Split('\n'))
						{
							if (directPingUrl.Length == 0) continue;
							_Logger.Verbose("SendLinkbackPingsForItem", "direct Pingback for Url " + directPingUrl);
							SendTrackbackPing(directPingUrl, localPostInfoToSubmit);
						}
					}
				}
			}
			catch (Exception ex)
			{
				// Direct trackback failed
				_Logger.Error("SendLinkbackPingsForItem", "Direct trackback failed: " + ex);
			}
			#endregion

			if (parameter.TheBlogSettings.PingbacksEnabled == false || postBodyHtml.Trim().Length == 0)
			{
				_Logger.Verbose("SendLinkbackPingsForItem", string.Format("PingbacksEnabled: {0}, postBodyHtml is empty", parameter.TheBlogSettings.PingbacksEnabled));
				return;
			}

			// Parse post body and attempt to ping each link using ProcessPermalink()
			XPathNavigator postBodyXhtmlNavigator;
			try
			{
				_Logger.Verbose("SendLinkbackPingsForItem", "creating XPathNavigator");
				HtmlNode postBodyHtmlNode = HtmlNode.CreateNode(postBodyHtml);
				postBodyXhtmlNavigator = postBodyHtmlNode.CreateNavigator();
			}
			catch
			{
				_Logger.Verbose("SendLinkbackPingsForItem", "postBody is unpareable");
				return; // Unparseable.. will not perform automatic pinging
			}

			var previouslyEncounteredPermalinks = new Dictionary<string, bool>();
			_Logger.Verbose("SendLinkbackPingsForItem", "Try to find pingback Urls form post body");
			FindUrlsFromPostBody(currentWeb, localPostInfoToSubmit, parameter, postBodyXhtmlNavigator, previouslyEncounteredPermalinks, encounteredPingUrls);
		}

		private void FindUrlsFromPostBody(SPWeb currentWeb, LocalPostInfo localPostInfoToSubmit, LinkbackThreadParameters parameter, XPathNavigator postBodyXhtmlNavigator, IDictionary<string, bool> previouslyEncounteredPermalinks, IDictionary<string, bool> encounteredPingUrls)
		{
			foreach (XPathNavigator hrefValue in postBodyXhtmlNavigator.Select("//a/@href"))
			{
				string potentialRemoteUrl = hrefValue.Value;
				_Logger.Verbose("FindUrlsFromPostBody", "potentialRemoteUrl: " + potentialRemoteUrl);
				try
				{
					if (string.IsNullOrEmpty(potentialRemoteUrl)) continue;

					//TODO: Check exclusions list before processing potential permalink
					if (potentialRemoteUrl.StartsWith("http", StringComparison.OrdinalIgnoreCase) == false)
					{
						if (potentialRemoteUrl.StartsWith(currentWeb.ServerRelativeUrl))
						{
							if (parameter.TheBlogSettings.DisableSelfTrackbacks)
							{
								_Logger.Verbose("FindUrlsFromPostBody", "self trackback. skipping url");
								continue; // relative link.. content is on the same site and can be skipped
							}
						}
					}
					if (parameter.TheBlogSettings.DisableSelfTrackbacks && potentialRemoteUrl.StartsWith(currentWeb.Url, StringComparison.OrdinalIgnoreCase))
					{
						_Logger.Verbose("FindUrlsFromPostBody", "skip links to content on the same site");
						continue; // skip links to content on the same site
					}

					// Check if hyperlink was already encountered in this blog post publish:
					string permalinkUniqueKey = potentialRemoteUrl.ToLower().TrimEnd('/');
					if (previouslyEncounteredPermalinks.ContainsKey(permalinkUniqueKey))
					{
						_Logger.Verbose("FindUrlsFromPostBody", "same hyperlink appears in post body twice");
						continue; // same hyperlink appears in post body twice
					}

					previouslyEncounteredPermalinks.Add(permalinkUniqueKey, true);

					if (Uri.IsWellFormedUriString(potentialRemoteUrl, UriKind.Relative))
					{
						potentialRemoteUrl = currentWeb.Site.Url + potentialRemoteUrl;
						_Logger.Verbose("FindUrlsFromPostBody", "potentialRemoteUrl is not " + potentialRemoteUrl);
					}

					ProcessPermalink(potentialRemoteUrl, localPostInfoToSubmit, encounteredPingUrls);
				}
				catch (Exception ex)
				{
					// an error occurred, but we'll try the next potential permalink
					try
					{
						_Logger.Error("FindUrlsFromPostBody", "Exception: " + ex);
						var linkbackDetails = new LinkbackDetails
														  {
															  LocalPostInfo = localPostInfoToSubmit,
															  LinkbackResultType = LinkbackResultType.Error,
															  Permalink = potentialRemoteUrl,
															  ResultDetails = string.Format("Unable to process the URL '{0}'. Please ensure that the link is valid and that you have access to the content at the URL (without entering credentials): {1}", potentialRemoteUrl, ex.Message)
														  };
						LogLinkbackDetails(linkbackDetails);
					}
					catch (Exception e)
					{
						_Logger.Error("FindUrlsFromPostBody", "LinkBackDetails failed: " + e);
					}
				}
			}
		}

		private void CreateTimerJob(SPListItem item)
		{
			_Logger.Verbose("CreateTimerJob", string.Format("The posts {0} published date is in the future. Need to deal with pings.", item[SPBuiltInFieldId.Title]));
			//var currentWeb = item.Web;
			// TODO figure another way. the app pool does not have permissions for the config_db (createing a webapp timer job)
			// impersonate as central adminsitration application pool user

			//using (var site = new SPSite(_CurrentWeb.Url))
			//{
			//    // now we can create jobs on the webapplication
			//    TrackbackJob trackbackJob = new TrackbackJob(site.WebApplication);
			//    trackbackJob.Settings = new TrackbackJobSettings(item, site.WebApplication);

			//    SPOneTimeSchedule schedule = new SPOneTimeSchedule();
			//    schedule.Time = ((DateTime) item[SPBuiltInFieldId.PublishedDate]).AddSeconds(1);
			//    trackbackJob.Schedule = schedule;
			//    trackbackJob.Update();
			//}
		}

		/// <summary>
		/// try to find a ping url from the url. if found, send ping
		/// </summary>
		/// <param name="remoteUrl"></param>
		/// <param name="localPostInfoToSubmit"></param>
		/// <param name="encounteredPingUrls"></param>
		private void ProcessPermalink(string remoteUrl, LocalPostInfo localPostInfoToSubmit, IDictionary<string, bool> encounteredPingUrls)
		{
			_Logger.Verbose("ProcessPermalink", "Need to find the ping URL in the posts HTML or response headers. URL: " + remoteUrl);
			// Need to find the ping URL in the posts HTML or response headers
			string remoteTrackbackUrl;
			string remotePingbackUrl;

			// Fetch page from remoteUrl:
			var pageRequest = (HttpWebRequest)WebRequest.Create(remoteUrl);
			pageRequest.AllowAutoRedirect = true;
			pageRequest.Accept = "text/html"; // accept only HTML

			LinkbackType linkbackType = LinkbackType.Trackback;
			using (var pageResponse = pageRequest.GetResponse() as HttpWebResponse)
			{
				if (pageResponse == null || pageResponse.StatusCode != HttpStatusCode.OK)
				{
					if (pageResponse != null) _Logger.Verbose("ProcessPermalink", "pageResponse not OK. HttpStatusCode is: " + pageResponse.StatusCode);
					else _Logger.Verbose("ProcessPermalink", "pageResponse is null");
					return;
				}

				var htmlDoc = new HtmlDocument();
				// Use HTML Agility pack to get XPathNavigator for blog post as XHTML:
				using (Stream responseStream = pageResponse.GetResponseStream())
				{
					_Logger.Verbose("ProcessPermalink", "parsing response Stream");
					htmlDoc.Load(responseStream);
				}
				_Logger.Verbose("ProcessPermalink", "creating XPathNavigator from response");
				XPathNavigator pageHtmlXpathNavigator = htmlDoc.CreateNavigator();

				// Attempt trackback first:
				remoteTrackbackUrl = FindTrackbackPingUrl(pageHtmlXpathNavigator);
				_Logger.Verbose("ProcessPermalink", "Trackback pingUrl: " + remoteTrackbackUrl);
				
				// Try find pingback ping URL:
				remotePingbackUrl = FindPingbackPingUrl(pageResponse, htmlDoc);
				_Logger.Verbose("ProcessPermalink", "Pingback pingUrl: " + remotePingbackUrl);
				if (string.IsNullOrEmpty(remoteTrackbackUrl))
				{
					// set type to pingback, if no trackback url was found
					linkbackType = LinkbackType.Pingback;
				}
			}

			#region validate remotePingUrl
			if (string.IsNullOrEmpty(remoteTrackbackUrl) && string.IsNullOrEmpty(remotePingbackUrl))
			{
				// Resource does not support either automated trackbacks or pingbacks
				_Logger.Information("ProcessPermalink", "Resource does not support either automated trackbacks or pingbacks");
				return;
			}

			remoteTrackbackUrl = FormatUrl(pageRequest, remoteTrackbackUrl);
			remotePingbackUrl = FormatUrl(pageRequest, remotePingbackUrl);

			if ((remoteTrackbackUrl != null && encounteredPingUrls.ContainsKey(remoteTrackbackUrl)) || 
				(remotePingbackUrl != null && encounteredPingUrls.ContainsKey(remotePingbackUrl)))
			{
				_Logger.Information("ProcessPermalink", "already pinged this trackback ping URL on the same post publish");
				return; // already pinged this trackback ping URL on the same post publish
			}
			#endregion

			if (!string.IsNullOrEmpty(remoteTrackbackUrl)) encounteredPingUrls.Add(remoteTrackbackUrl, true);
			if (!string.IsNullOrEmpty(remotePingbackUrl)) encounteredPingUrls.Add(remotePingbackUrl, true);

			var linkbackDetails = new LinkbackDetails();
			if (linkbackType == LinkbackType.Trackback)
			{
				linkbackDetails = SendTrackbackPing(remoteTrackbackUrl, localPostInfoToSubmit, remoteUrl);
			}
			// try to send pingback if trackback failed, or if there was no trackback url
			if (linkbackType == LinkbackType.Pingback || linkbackDetails.LinkbackResultType != LinkbackResultType.Success)
			{
				if (LinkbackDetails(localPostInfoToSubmit, remoteUrl, remotePingbackUrl, LinkbackType.Pingback, out linkbackDetails) == false)
				{
					_Logger.Information("ProcessPermalink", "Already send pingback request to " + remotePingbackUrl);
				}
				else
				{
					// now try to pingback. change the url, because it might be the trackback url
					linkbackDetails.PingUrl = remotePingbackUrl;
					SendLinkbackPing(linkbackDetails);
				}
			}
			// log after both methods have been tried
			LogLinkbackDetails(linkbackDetails);
		}

		/// <summary>
		/// make the url absolute, if it is relative
		/// </summary>
		/// <param name="pageRequest"></param>
		/// <param name="url"></param>
		/// <returns></returns>
		private static string FormatUrl(HttpWebRequest pageRequest, string url)
		{
			if (Uri.IsWellFormedUriString(url, UriKind.Relative))
			{
				string tempUrl = pageRequest.Address.AbsoluteUri;
				url = tempUrl.Substring(0, tempUrl.IndexOf("/", 8)) + url;
			}
			return url;
		}

		private string FindTrackbackPingUrl(XPathNavigator pageHtmlXpathNavigator)
		{
			string pingUrl;

			// Try to obtain ping URL from link tag: <link rel="trackback" href="{ping url}">
			XPathNavigator linkTagXpathNavigator = pageHtmlXpathNavigator.SelectSingleNode(@"//head/link[@rel=""trackback""]");
			if (linkTagXpathNavigator != null)
			{
				pingUrl = linkTagXpathNavigator.GetAttribute("href", linkTagXpathNavigator.NamespaceURI);
				if (string.IsNullOrEmpty(pingUrl) == false)
				{
					_Logger.Verbose("FindTrackbackPingUrl", "Got pingUrl from head: " + pingUrl);
					return pingUrl;
				}
			}

			// Wordpress uses a non-standard <a> tag with rel="trackback" to expose the trackback ping URL.
			// This is probably meant for manual trackback ping, but we can automate it anyways
			XPathNavigator aTagRelTrackbackXPathNavigator = pageHtmlXpathNavigator.SelectSingleNode(@"//a[@rel=""trackback""]");
			if (aTagRelTrackbackXPathNavigator != null)
			{
				pingUrl = aTagRelTrackbackXPathNavigator.GetAttribute("href", aTagRelTrackbackXPathNavigator.NamespaceURI);
				if (string.IsNullOrEmpty(pingUrl) == false)
				{
					_Logger.Verbose("FindTrackbackPingUrl", "Got pingUrl from wordpress: " + pingUrl);
					return pingUrl;
				}
			}

			// Still not found. Proceed by looking for ping URL in RDF within HTML body (will find and ping the first - i.e. latest - post if the URL is not a permalink)
			var pageHtmlXmlnsManager = new XmlNamespaceManager(pageHtmlXpathNavigator.NameTable);
			pageHtmlXmlnsManager.AddNamespace("rdf", Globals.RdfXmlnsUri);
			foreach (XPathNavigator rdfXpathNavigator in pageHtmlXpathNavigator.Select("//rdf:RDF/rdf:Description", pageHtmlXmlnsManager))
			{
				pingUrl = rdfXpathNavigator.GetAttribute("ping", Globals.TrackbackRdfXmlnsUri);
				if (string.IsNullOrEmpty(pingUrl) == false)
				{
					_Logger.Verbose("FindTrackbackPingUrl", "Got pingUrl from RDF description ping: "+pingUrl);
					return pingUrl;
				}
				// continue and check next rdf:description node for a trackback:ping attribute
				pingUrl = rdfXpathNavigator.GetAttribute("trackback:ping", Globals.TrackbackRdfXmlnsUri);
				if (string.IsNullOrEmpty(pingUrl) == false)
				{
					_Logger.Verbose("FindTrackbackPingUrl", "Got pingUrl from RDF description trackback:ping: "+pingUrl);
					return pingUrl;
				}
			}

			// try to find the trackback url from a comment in the body
			foreach (XPathNavigator node in pageHtmlXpathNavigator.Select("//comment()", pageHtmlXmlnsManager))
			{
				int index = node.Value.IndexOf("trackback", StringComparison.CurrentCultureIgnoreCase);
				if (index == -1) continue;

				// load inner xml into an XmlDocument to find the trackback url
				string innerComment = node.Value.Replace("<!--", "").Replace("-->", "").Trim(new[] {'\r', '\n', ' '});
				var xdoc = new XmlDocument();
				xdoc.LoadXml(innerComment);
				XmlNode xmlNode = xdoc.SelectSingleNode("//rdf:RDF/rdf:Description", pageHtmlXmlnsManager);
				XmlAttribute attribute = xmlNode.Attributes["ping"];
				if (attribute != null && !string.IsNullOrEmpty(attribute.Value))
				{
					_Logger.Verbose("FindTrackbackPingUrl", "Got pingUrl from comment RDF description ping: " + attribute.Value);
					return attribute.Value;
				}

				attribute = xmlNode.Attributes["trackback:ping"];
				if (attribute != null && !string.IsNullOrEmpty(attribute.Value))
				{
					_Logger.Verbose("FindTrackbackPingUrl", "Got pingUrl from comment RDF description trackback:ping: " + attribute.Value);
					return attribute.Value;
				}
			}

			return null; // failed to find ping URL at this point
		}

		private string FindPingbackPingUrl(WebResponse response, HtmlDocument htmlDocument)
		{
			int pingUrlKeyIndex = Array.FindIndex(response.Headers.AllKeys,
															  k => k.Equals("x-pingback", StringComparison.OrdinalIgnoreCase) ||
																	 k.Equals("pingback", StringComparison.OrdinalIgnoreCase));

			string pingUrl = null;
			if (pingUrlKeyIndex != -1)
			{
				pingUrl = response.Headers[pingUrlKeyIndex];
				_Logger.Verbose("FindPingbackPingUrl", "Got PingbackUrl from response Header: " + pingUrl);
			}
			if (string.IsNullOrEmpty(pingUrl))
			{
				// try to find pingback url from link
				var regex = new Regex("<link rel=\"pingback\" href=\"(?<linkbackurl>[^\"]+)\" ?/?>",
											  RegexOptions.Compiled | RegexOptions.ExplicitCapture | RegexOptions.Singleline);
				MatchCollection matches = regex.Matches(htmlDocument.DocumentNode.InnerHtml);
				foreach (Match match in matches)
				{
					pingUrl = match.Groups["linkbackurl"].Value;
					_Logger.Verbose("FindPingbackPingUrl", "Got PingbackUrl from link element: " + pingUrl);
				}
			}
			return pingUrl;
		}

		private void SendTrackbackPing(string trackbackPingUrl, LocalPostInfo localPostInfoToSubmit)
		{
			SendTrackbackPing(trackbackPingUrl, localPostInfoToSubmit, null);
		}

		private LinkbackDetails SendTrackbackPing(string trackbackPingUrl, LocalPostInfo localPostInfoToSubmit, string permalink)
		{
			LinkbackDetails linkbackDetails;
			if (LinkbackDetails(localPostInfoToSubmit, permalink, trackbackPingUrl, LinkbackType.Trackback, out linkbackDetails) == false)
			{
				_Logger.Information("SendTrackbackPing", "Already send pingback request to " + trackbackPingUrl);
				linkbackDetails.LinkbackResultType = LinkbackResultType.AlreadySentLinkback;
				return linkbackDetails;
			}

			try
			{
				ServicePointManager.Expect100Continue = false;
				byte[] trackbackPingPostBytes = localPostInfoToSubmit.GetBytesForTrackbackPingHttpPost();

				_Logger.Verbose("SendTrackbackPing", "Creating webrequest for url: " + trackbackPingUrl);
				var webRequest = (HttpWebRequest)WebRequest.Create(trackbackPingUrl);
				webRequest.Method = "POST";
				webRequest.ProtocolVersion = HttpVersion.Version10;
				webRequest.KeepAlive = false;
				webRequest.ContentType = "application/x-www-form-urlencoded";
				webRequest.ContentLength = trackbackPingPostBytes.LongLength;
				webRequest.MaximumAutomaticRedirections = 4;
				webRequest.Timeout = 20000; // give at most 20 seconds to receive ping response
				webRequest.ReadWriteTimeout = 20000;
				webRequest.ServicePoint.MaxIdleTime = 20000;

				// Add the bytes for the outbound stream:
				Stream requestStream = webRequest.GetRequestStream();
				try
				{
					_Logger.Verbose("SendTrackbackPing", "Adding request stream");
					requestStream.Write(trackbackPingPostBytes, 0, trackbackPingPostBytes.Length);
					try
					{
						RequestSuccessfull(webRequest, linkbackDetails);
					}
					catch (WebException ex)
					{
						if (ex.Status == WebExceptionStatus.KeepAliveFailure)
						{
							// give it a second chance
							RequestSuccessfull(webRequest, linkbackDetails);
						}
						else
						{
							linkbackDetails.LinkbackResultType = LinkbackResultType.Error;
							linkbackDetails.ResultDetails = ex.Message;
							//LogLinkbackDetails(linkbackDetails);
							_Logger.Error("SendTrackbackPing", "WebException: " + ex);
							return linkbackDetails;
						}
					}
				}
				finally
				{
					requestStream.Close();
				}
			}
			catch (Exception ex)
			{
				linkbackDetails.LinkbackResultType = LinkbackResultType.Unknown;
				linkbackDetails.ResultDetails = "An unknown error occurred.: " + ex.Message;
				//LogLinkbackDetails(linkbackDetails);
				_Logger.Error("SendTrackbackPing", "Exception: " + ex);
				return linkbackDetails;
			}
			return linkbackDetails;
		}

		/// <summary>
		/// Sends pingbacks to the targetUrl.
		/// </summary>
		private void SendLinkbackPing(LinkbackDetails details)
		{
			if (details.LocalPostInfo.Url == null || details.Permalink == null) return;

			try
			{
				Uri url;
				if (!string.IsNullOrEmpty(details.PingUrl) && Uri.TryCreate(details.PingUrl, UriKind.Absolute, out url))
				{
					var proxy = (IPingbackPing)XmlRpcProxyGen.Create(typeof(IPingbackPing));
					proxy.Url = url.ToString();

					string result = proxy.Ping(details.LocalPostInfo.Url, details.Permalink);

					details.LinkbackResultType = LinkbackResultType.Success;
					details.ResultDetails = "The other blog responded with the following: " + result;
					//LogLinkbackDetails(details);
				}
			}
			catch (XmlRpcFaultException ex)
			{
				switch (ex.FaultCode)
				{
					case 48:
						details.ResultDetails = "The pingback has already been registered.";
						details.LinkbackResultType = LinkbackResultType.Success;
						break;
					default:
						details.ResultDetails = ex.FaultString;
						details.LinkbackResultType = LinkbackResultType.Error;
						break;
				}
				//LogLinkbackDetails(details);
				return;
			}
			catch (Exception ex)
			{
				details.LinkbackResultType = LinkbackResultType.Error;
				details.ResultDetails = "An error occurred: " + ex.Message;
				//LogLinkbackDetails(details);
				_Logger.Error("SendLinkbackPing", ex.ToString());
				return;
			}
		}

		/// <summary>
		/// create object for trackback/linkback result
		/// </summary>
		/// <param name="localPostInfoToSubmit"></param>
		/// <param name="permalink"></param>
		/// <param name="trackbackPingUrl"></param>
		/// <param name="type"></param>
		/// <param name="details"></param>
		/// <returns>false if url is malformed or already pinged</returns>
		private bool LinkbackDetails(LocalPostInfo localPostInfoToSubmit, string permalink, string trackbackPingUrl, LinkbackType type, out LinkbackDetails details)
		{
			details = new LinkbackDetails
			{
				LocalPostInfo = localPostInfoToSubmit,
				LinkbackType = type,
				LinkbackResultType = LinkbackResultType.Unknown,
				Permalink = permalink,
				PingUrl = trackbackPingUrl
			};
			_Logger.Verbose("LinkbackDetails", "LinkbackDetails: " + details);

			if (Uri.IsWellFormedUriString(details.PingUrl, UriKind.Absolute) == false)
			{
				details.LinkbackResultType = LinkbackResultType.Error;
				details.ResultDetails = "Invalid trackback ping URL";
				_Logger.Verbose("LinkbackDetails", "ResultDetails: " + details.ResultDetails);
				LogLinkbackDetails(details);
				return false;
			}

			if (IsDuplicatedPing(details))
			{
				_Logger.Verbose("LinkbackDetails", "linkback Url is duplicate. Don't pingback");
				return false;
			}
			return true;
		}

		private bool IsDuplicatedPing(LinkbackDetails linkbackDetails)
		{
			var query = new SPQuery
								 {
									 Query = string.Format(SearchLinkbackHistoryListCamlQueryFormat,
																  linkbackDetails.LocalPostInfo.ItemId,
																  XmlEscape(linkbackDetails.PingUrl.ToLower())),
									 RowLimit = 1
								 };

			using (var site = new SPSite(linkbackDetails.LocalPostInfo.WebUrl))
			using (SPWeb web = site.OpenWeb())
			{
				SPList linkbacksSentList = null;
				try
				{
					linkbacksSentList = web.Lists[Globals.LinkbackHistoryListTitle];
				}
				catch (Exception ex)
				{
					_Logger.Verbose("IsDuplicatePing", "LinkbackHistoryListTitle invalid: " + ex);
				}
				if (linkbacksSentList == null)
					return false; // without the list, assume the ping is not a duplicate

				SPListItemCollection resultItems = linkbacksSentList.GetItems(query);
				if (resultItems != null && resultItems.Count > 0)
				{
					_Logger.Verbose("LinkbackDetails", "already pinged");
					return true; // already pinged for this post
				}
			}
			return false; // not already pinged
		}

		private static void LogLinkbackDetails(LinkbackDetails linkbackDetails)
		{
			string batchAddResult = string.Empty;
			try
			{
				// user must be contributor, so elevation not required:
				using (var site = new SPSite(linkbackDetails.LocalPostInfo.WebUrl))
				using (SPWeb web = site.OpenWeb())
				{
					// if there are many items, list.items.add can be slow
					SPList linkbackHistoryList = web.Lists[Globals.LinkbackHistoryListTitle];
					string linkbackHistoryListId = linkbackHistoryList.ID.ToString();
					string addLinkbackHistoryCamlBatch = string.Format(AddLinkbackHistoryCamlBatchFormat,
																						linkbackHistoryListId,
																						XmlEscape("Sent " + linkbackDetails.LinkbackType),
																						linkbackDetails.LocalPostInfo.ItemId,
																						linkbackDetails.LinkbackType,
																						XmlEscape(linkbackDetails.Permalink).ToLower(),
																						XmlEscape(linkbackDetails.PingUrl).ToLower(),
																						linkbackDetails.LinkbackResultType,
																						XmlEscape(linkbackDetails.ResultDetails));
					batchAddResult = web.ProcessBatchData(addLinkbackHistoryCamlBatch);
				}
			}
			catch (Exception ex)
			{
				Trace.WriteLine(string.Format("LogLinkbackDetails error: {0} (ProcessBatchData Result: {1})", ex, batchAddResult));
			}
		}

		private static string GetExcerpt(string text)
		{
			if (text.Length <= MaxExcerptLengthSubmittedWithLinkback)
				return text;
			return text.Substring(0 /*start*/, MaxExcerptLengthSubmittedWithLinkback - 4 /*length*/) + " ...";
		}

		private static string XmlEscape(string text)
		{
			if (string.IsNullOrEmpty(text))
				return string.Empty;
			return text.Replace("&", "&amp;").Replace(">", "&gt;").Replace("<", "&lt;");
		}

		#endregion

		/// <summary>
		/// check response from foreign blog and log details
		/// </summary>
		/// <param name="request"></param>
		/// <param name="details"></param>
		/// <returns></returns>
		private void RequestSuccessfull(HttpWebRequest request, LinkbackDetails details)
		{
			XmlReader reader;
			using (var webResponse = (HttpWebResponse)request.GetResponse())
			{
				if (request.HaveResponse == false || webResponse.ContentLength == 0L)
				{
					// Assume error if no response is received:
					details.LinkbackResultType = LinkbackResultType.Error;
					details.ResultDetails = "No response to trackback ping received from other blog.";
					_Logger.Verbose("RequestSuccessfull", details.ResultDetails);
					LogLinkbackDetails(details);
					return;
				}

				// If a content type value is present but it is not XML, then the response is invalid
				if (string.IsNullOrEmpty(webResponse.ContentType) == false && webResponse.ContentType.Contains("xml") == false)
				{
					details.LinkbackResultType = LinkbackResultType.Error;
					details.ResultDetails = "Invalid response to trackback ping received from other blog. Response is not xml.";
					_Logger.Verbose("RequestSuccessfull", details.ResultDetails);
					LogLinkbackDetails(details);
					return;
				}

				using (Stream responseStream = webResponse.GetResponseStream())
				{
					var settings = new XmlReaderSettings { ProhibitDtd = false };
					_Logger.Verbose("RequestSuccessfull", "Creating reader");
					reader = XmlReader.Create(responseStream, settings);
				}
			}

			using (reader)
			{
				reader.MoveToContent();

				if (reader.IsStartElement("response") == false || reader.ReadToFollowing("error") == false)
				{
					// Encountered HTML or some other invalid response
					details.LinkbackResultType = LinkbackResultType.Unknown;
					details.ResultDetails = "Invalid response to trackback ping received from other blog.";
					_Logger.Verbose("RequestSuccessfull", details.ResultDetails);
					LogLinkbackDetails(details);
					return;
				}

				string errCode = reader.ReadString().Trim();
				if (errCode == "0")
				{
					details.LinkbackResultType = LinkbackResultType.Success;
					_Logger.Verbose("RequestSuccessfull", "Request successfull: " + details.ResultDetails);
				}
				else
				{
					details.LinkbackResultType = LinkbackResultType.Error;
					string errorMessage;
					try
					{
						errorMessage = reader.ReadToFollowing("message") ? reader.ReadString() : "Unspecified error.";
					}
					catch
					{
						errorMessage = "Unspecified error.";
					}
					details.ResultDetails = string.Format("The other blog responded with the following error: \"{0}\" (Error Code '{1}').", errorMessage, errCode);
					_Logger.Verbose("RequestSuccessfull", details.ResultDetails);
				}
			}

			_Logger.Verbose("RequestSuccessfull", details.ResultDetails);
			LogLinkbackDetails(details);
			return;
		}
	}
}